{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataset import DeepFashionDataset\n",
    "import torch \n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import transforms\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = DeepFashionDataset('../deepfashion-multimodal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = DataLoader(data_train,1,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 3, 224, 224]) ('The sweater this female wears has long sleeves and it is with cotton fabric and graphic patterns. The neckline of the sweater is crew. This female wears a three-point shorts, with cotton fabric and pure color patterns. The female also wears an outer clothing, with leather fabric and pure color patterns. There is a hat in her head. There is a ring on her finger. There is an accessory on her wrist. There is an accessory in his her neck.',)\n"
     ]
    }
   ],
   "source": [
    "for i in train_loader:\n",
    "    print(i[0].shape,i[1])\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VisionTransformer(\n",
      "  (patch_embed): PatchEmbed(\n",
      "    (proj): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))\n",
      "    (norm): Identity()\n",
      "  )\n",
      "  (pos_drop): Dropout(p=0.0, inplace=False)\n",
      "  (patch_drop): Identity()\n",
      "  (norm_pre): Identity()\n",
      "  (blocks): Sequential(\n",
      "    (0): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "    (1): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "    (2): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "    (3): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "    (4): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "    (5): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "    (6): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "    (7): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "    (8): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "    (9): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "    (10): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "    (11): Block(\n",
      "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "        (q_norm): Identity()\n",
      "        (k_norm): Identity()\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls1): Identity()\n",
      "      (drop_path1): Identity()\n",
      "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (drop1): Dropout(p=0.0, inplace=False)\n",
      "        (norm): Identity()\n",
      "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        (drop2): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (ls2): Identity()\n",
      "      (drop_path2): Identity()\n",
      "    )\n",
      "  )\n",
      "  (norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "  (fc_norm): Identity()\n",
      "  (head_drop): Dropout(p=0.0, inplace=False)\n",
      "  (head): Sequential()\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "import timm\n",
    "import torch \n",
    "from torch import nn\n",
    "model_name = 'vit_base_patch16_384'\n",
    "model = timm.create_model(model_name)\n",
    "model.head =nn.Sequential()\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "img =torch.ones(1,3,384,384)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
